import scrapy
from scrapy_splash import SplashRequest

from zhihu.items import ZhihuPostItem
class PostSpider(scrapy.Spider):
    name='postSpider'
    allowed_domains=['www.zhihu.com']
    start_urls=['https://www.zhihu.com/people/duo-kan-le-ji-yan/posts']

    def start_requests(self):
        splah_args = {
            "lua_source": """
            function main(splash, args)
              assert(splash:go(args.url))
              assert(splash:wait(5))
              return {
                html = splash:html(),
                png = splash:png(),
                har = splash:har(),
              }
            end
            """
        }
        headers = {
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
                          'Chrome/72.0.3626.109 Safari/537.36',
        }
        yield SplashRequest(url="https://www.zhihu.com/people/duo-kan-le-ji-yan/posts", callback=self.parse, args=splah_args,
                            headers=headers)
    def parse(self,response):
        print(response.xpath("//body").extract())
        subSelector = response.xpath('//h2[@class="ContentItem-title"]')
        items = []
        print(len(subSelector))
        for sub in subSelector:
            print(sub.extract())
            item = ZhihuPostItem()
            item['title'] = sub.xpath('./a/text()').extract()
            items.append(item)
        return items
